﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.IO;

namespace Co_location
{
    class Transportation
    {
        static double timethred = 10;

        static double disthred = 0.0002;
        static double gracePeriod = 60; // in seconds
        static double ignorePeriod = 30; // in seconds
        
        #region for the external demo of nakuhi
        //static string outputPath = @"D:\ACM_Class\AAA\SilverlightApplication2\SilverlightApplication2\crop_area.csv";
        //static int TotalNumber = 200;
        //static int TempNumber = 0;
        #endregion       

        private static string root = System.IO.Directory.GetCurrentDirectory();

        private static Dictionary<int, List<int>> tf;
        private static Dictionary<int, Dictionary<int, int>> label_tf = new Dictionary<int, Dictionary<int, int>>(); //<userid,<lable index, label count>>
        private static Dictionary<int, List<int>> user2LabelVector = new Dictionary<int, List<int>>(); //<userid,<label index, cumulated seconds>>
        // the vector length is doubled due to the representation of weekday and weekend
        private static Dictionary<int, List<int>> user2LabelVectorWW = new Dictionary<int, List<int>>();
        private static double[] idf;

        private static LinkedList<string> labels = new LinkedList<string>();
        private static int labelCount = 0;
        private static Dictionary<string, string> labelHashTable = new Dictionary<string, string>() { 
        { "bike", "bike" }, { "taxi", "car" }, { "walk", "walk" }, { "car", "car" }, { "subway", "subway" }, { "bus", "bus" }, 
        { "motor", "moto" }, { "train", "train" }, { "run", "run" }, { "driving meet conjestion", "car" }, 
        { "plane", "plane" }, { "moto", "moto" }, { "airplane", "plane" }, { "railway", "railway" }, { "boat", "boat" }, 
        { "motocycle", "moto" }, { "other", "other" } };

        private static Dictionary<string, int> label2VectorIndex = new Dictionary<string, int>();

        private static Dictionary<int, Dictionary<int, double>> WSCSMatrix;
        private static HashSet<DayOfWeek> WEEKENDS = new HashSet<DayOfWeek> { DayOfWeek.Saturday, DayOfWeek.Sunday };

        private static int featVecLen = 15;

        #region Parameters To Be Setup
        private static double knownRatio = 0.25;
        private static bool matrixUseWeekend = false;
        #endregion

        public static void Setup(string[] args)
        {
            int idx = 0;
            while (idx < args.Length && args[idx].StartsWith("-"))
            {
                string p = args[idx].ToLower().Substring(1);
                if (p == "knownratio")
                    knownRatio = double.Parse(args[++idx]);
                if (p == "weekend")
                    matrixUseWeekend = bool.Parse(args[++idx]);
            }
        }

        #region Transportation

        public static void ReadTransportation()
        {
            DateTime ST = DateTime.Now;
            MCBConsole.Trace("Start : " + ST);
            DateTime NOW;
            TimeSpan dur;
            
            List<string> subFolders = getTransSubFoleders();
            string path = "";

            int count = 0, total = 0;

            SortedDictionary<int, SortedDictionary<DateTime, Position>> allUserPositions = new SortedDictionary<int, SortedDictionary<DateTime, Position>>();

            //StreamWriter writer = new StreamWriter(outputPath);
            foreach (string subfolder in subFolders)
            {
                path = root + "\\" + subfolder + "\\trajectory";
                IEnumerable<string> files = System.IO.Directory.EnumerateFiles(path);
                IEnumerator<string> fEnum = files.GetEnumerator();

                double latitude;
                double longitude;
                int code;
                double altitude;
                double date;
                DateTime datetime;

                SortedDictionary<DateTime, Position> singleUserTrace = new SortedDictionary<DateTime, Position>();
                SortedDictionary<DateTime, double> singleUserSpeed = new SortedDictionary<DateTime, double>();
                
                //TempNumber = 0;               

                while (fEnum.MoveNext())
                {
                    string filename = fEnum.Current;

                    StreamReader reader = new StreamReader(filename);

                    string line = "";

                    while ((line = reader.ReadLine()) != null)
                    {
                        string[] splits = line.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);

                        if (splits.Length != 7)
                            continue;

                        if (!double.TryParse(splits[0], out latitude))
                            continue;
                        if (!double.TryParse(splits[1], out longitude))
                            continue;
                        if (!int.TryParse(splits[2], out code))
                            continue;
                        if (!double.TryParse(splits[3], out altitude))
                            continue;
                        if (!double.TryParse(splits[4], out date))
                            continue;
                        if (!parseDate(splits[5], splits[6], out datetime))
                        {
                            MCBConsole.Trace(filename);
                            Console.ReadLine();
                            continue;
                        }

                        Position p;
                        if (!singleUserTrace.TryGetValue(datetime, out p))
                        {
                            p =  new Position(latitude, longitude, altitude, code == 0);                            

                            singleUserTrace.Add(datetime,p);
                            count++;

                            //if (TempNumber < TotalNumber && latitude > 39.82 && latitude < 40 && longitude > 116.28 && longitude < 116.42)
                            //{
                            //    writer.WriteLine(line);
                            //    TempNumber++;
                            //}
                        }
                    }
                    reader.Close();                    
                }               

                allUserPositions.Add(int.Parse(subfolder), singleUserTrace);
               
                NOW = DateTime.Now;
                dur = ST - NOW;
                MCBConsole.Trace(subfolder + " count: " + count + "  " + dur.ToString());
                total += count;
                count = 0;
            }
            //writer.Close();
            MCBConsole.Trace("Total : " + total);
            //computeDistance(allUserPositions);

            calcSpeed(allUserPositions);           

            NOW = DateTime.Now;
            dur = NOW - ST;
            MCBConsole.Trace("Calc Speed :" + dur.ToString());
            OutputSpeed();
            MCBConsole.Trace("Output Speed : " + (DateTime.Now - ST).ToString());
        }

        private static SortedDictionary<int, SortedDictionary<DateTime, SortedList<DateTime, double>>>
           allUserSpeed = new SortedDictionary<int, SortedDictionary<DateTime, SortedList<DateTime, double>>>();

        private static void calcSpeed(SortedDictionary<int, SortedDictionary<DateTime, Position>> allUserPositions)
        {
            foreach (int user in allUserPositions.Keys)
            {
                SortedDictionary<DateTime, Position> singleUserTrace = allUserPositions[user];
                allUserSpeed.Add(user, new SortedDictionary<DateTime, SortedList<DateTime, double>>());
                DateTime lastTime = singleUserTrace.First().Key, listStartTime = lastTime;
                Position lastPos = singleUserTrace.First().Value, pos;
                
                double speed = -1;
                foreach (DateTime time in singleUserTrace.Keys)
                {
                    pos = singleUserTrace[time];
                    if (pos.normal)
                    {
                        speed = CalcDistance(lastPos, pos) / (time - lastTime).TotalSeconds;
                        allUserSpeed[user][listStartTime].Add(time, speed);
                    }
                    else
                    {
                        allUserSpeed[user].Add(time, new SortedList<DateTime, double>());
                        listStartTime = time;
                    }
                    lastPos = pos;
                    lastTime = time;
                }                
            }
        }

        private static SortedDictionary<int, SortedDictionary<int, SortedDictionary<int, SortedList<DateTime, LabelInfo>>>>
            allUserLabel = new SortedDictionary<int, SortedDictionary<int, SortedDictionary<int, SortedList<DateTime, LabelInfo>>>>();

        public static void ReadTransLabel()
        {
            MCBConsole.DisableOutput();

            DateTime ST = DateTime.Now;
            MCBConsole.Trace("Start at " + ST);

            List<string> subFolders = getTransSubFoleders();

            #region prepare label vector

            foreach (string l in labelHashTable.Values)
                if (!labels.Contains(l)) labels.AddLast(l);

            MCBConsole.Trace("Label Count : " + labels.Count);
            labelCount = labels.Count;
            
            int v = 0;
            foreach (string l in labels)
            {
                label2VectorIndex.Add(l, v);
                v++;
            }

            foreach (string subFolder in subFolders)
            {
                int userID = int.Parse(subFolder);
                user2LabelVector.Add(userID, new List<int>());
                user2LabelVectorWW.Add(userID, new List<int>());
                label_tf.Add(userID, new Dictionary<int, int>());
                foreach (string l in labels)
                {
                    user2LabelVector[userID].Add(0);
                    user2LabelVectorWW[userID].Add(0);
                    user2LabelVectorWW[userID].Add(0);
                    label_tf[userID].Add(label2VectorIndex[l], 0);
                }
            }

            #endregion

            #region read in label           

            string path = "";

            int minDur = 100000, maxDur = 1;
            foreach (string subFolder in subFolders)
            {
                int userID = int.Parse(subFolder);
                allUserLabel.Add(userID, new SortedDictionary<int, SortedDictionary<int, SortedList<DateTime, LabelInfo>>>());

                path = root + "\\" + subFolder + "\\labels.txt";
                StreamReader reader = new StreamReader(path);
                string line;

                reader.ReadLine(); // the first line is schema
                while ((line = reader.ReadLine()) != null)
                {
                    string[] splits = line.Split(new char[] { '\t' }, StringSplitOptions.RemoveEmptyEntries);
                    if (splits.Length != 4)
                    {
                        MCBConsole.Warn("W0 "+path + "\n" + line + "\n");
                        continue;
                    }

                    int year, month, day;
                    int sHour, sMinute, sSecond;
                    int eHour, eMinute, eSecond;

                    try
                    {
                        parseLabelDate(splits[0], out year, out month, out day);
                        parseLabelTime(splits[1], out sHour, out sMinute, out sSecond);
                        parseLabelTime(splits[2], out eHour, out eMinute, out eSecond);
                    }
                    catch (Exception)
                    {
                        MCBConsole.Warn(path + "\n" + line + "\n");
                        continue;
                    }

                    DateTime startTime = new DateTime(year, month, day, sHour, sMinute, sSecond);                    
                    DateTime endTime = new DateTime(year, month, day, eHour, eMinute, eSecond);
                    if (sHour > 21 && eHour < 3)
                        endTime = endTime.AddDays(1);
                    int durationSeconds = (int)((endTime - startTime).TotalSeconds);

                    //save one
                    if (durationSeconds < 0 && Math.Abs(durationSeconds) < 3600) durationSeconds = -durationSeconds;

                    if (durationSeconds < 0) { MCBConsole.Warn("W1 "+line); continue; }

                    if (!allUserLabel[userID].Keys.Contains(year))
                        allUserLabel[userID].Add(year, new SortedDictionary<int, SortedList<DateTime, LabelInfo>>());
                    if (!allUserLabel[userID][year].Keys.Contains(month))
                        allUserLabel[userID][year].Add(month, new SortedList<DateTime, LabelInfo>());
                    if (allUserLabel[userID][year][month].Keys.Contains(startTime)) { MCBConsole.Warn("W2 "+line); continue; }
                    
                    if (durationSeconds < minDur) minDur = durationSeconds;
                    if (durationSeconds > maxDur) maxDur = durationSeconds;                   
                    
                    string label = splits[3];

                    int vIndex = label2VectorIndex[labelHashTable[label]];
                    if (WEEKENDS.Contains(startTime.DayOfWeek))
                        user2LabelVectorWW[userID][vIndex + labelCount] += durationSeconds;
                    else
                        user2LabelVectorWW[userID][vIndex] += durationSeconds;

                    user2LabelVector[userID][vIndex] += durationSeconds;
                    label_tf[userID][vIndex]++;
                    
                    allUserLabel[userID][year][month].Add(startTime, new LabelInfo(startTime, endTime, label));                    
                }
                reader.Close();
            }
            MCBConsole.Trace("Min Duration : " + minDur + "\t Max Duration : " + maxDur);
            #endregion

            outputLabelVector();

            if (matrixUseWeekend)
                calcWSCSofLabelVector(user2LabelVectorWW);
            else
                calcWSCSofLabelVector(user2LabelVector);

            outputWSCSMatrix();
        }

        private static void outputLabelVector()
        {
            StreamWriter writer = new StreamWriter(root + "\\labelVectors.txt");         

            foreach (KeyValuePair<int, List<int>> pair in user2LabelVector)
            {
                List<int> vector = pair.Value;
                foreach (int value in vector)
                {
                    writer.Write(value + "\t");
                }
                writer.WriteLine();
            }
            writer.Close();

            writer = new StreamWriter(root + "\\labelVectorsWW.txt");
            foreach (KeyValuePair<int, List<int>> pair in user2LabelVectorWW)
            {
                List<int> vector = pair.Value;
                foreach (int value in vector)
                    writer.Write(value + "\t");
                writer.WriteLine();
            }
            writer.Close();
        }

        private static void calcWSCSofLabelVector(Dictionary<int,List<int>> inLabelVector)
        {
            #region calc tf

            tf = new Dictionary<int, List<int>>();
            foreach (int user in inLabelVector.Keys)
            {         
                tf.Add(user, new List<int>());
                foreach (int v in inLabelVector[user])
                {
                    if (double.IsNaN(v))
                        throw new Exception();
                    tf[user].Add(v);
                }
            }
            #endregion

            #region calc idf
            int idfL = inLabelVector.Values.First().Count;

            idf = new double[idfL];
            for (int lbl = 0; lbl < idf.Length; lbl++)
            {
                double div = 0;
                foreach (int user in tf.Keys)
                {
                    if (tf[user][lbl] != 0)
                        div++;
                }
                idf[lbl] = Math.Log((double)tf.Keys.Count / div);
                if (double.IsNaN(idf[lbl]))
                    throw new Exception();
            }
            #endregion

            #region calc wscs matrix
            WSCSMatrix = new Dictionary<int, Dictionary<int, double>>();
            foreach (int usera in tf.Keys)
            {
                List<double> vectora = new List<double>();
                for (int i = 0; i < labels.Count; i++)
                {                    
                    vectora.Add(tf[usera][i] * idf[i]);
                }

                WSCSMatrix.Add(usera, new Dictionary<int, double>());
                foreach (int userb in tf.Keys)
                {
                    List<double> vectorb = new List<double>();
                    for (int i = 0; i < labels.Count; i++)
                    {
                        vectorb.Add(tf[userb][i] * idf[i]);                 
                    }

                    WSCSMatrix[usera].Add(userb, cosineSimilarity(vectora, vectorb));
                }
                WSCSMatrix[usera][usera] = 0;
            }
            #endregion
        }

        private static void outputWSCSMatrix()
        {
            StreamWriter writer = new StreamWriter(root + "\\WSCSMatrix.csv");
            
            foreach (int user in user2LabelVector.Keys)
                writer.Write("," + user);
            writer.WriteLine();

            foreach (int usera in user2LabelVector.Keys)
            {
                writer.Write(usera);
                foreach (int userb in user2LabelVector.Keys)
                {
                    double v = WSCSMatrix[usera][userb];
                    if (v < 0.4) v = 0;
                    writer.Write("," + v.ToString("0.000"));
                }
                writer.WriteLine();
            }
            writer.Close();
        }
        
        private static bool parseDate(string in_date, string in_time, out DateTime date)
        {
            string[] dates = in_date.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
            string[] times = in_time.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);

            int year, month, day;
            int hour, minute, second;
            date = DateTime.Now;

            if (dates.Length != 3)
            {
                Console.WriteLine("Date Format Exception: " + in_date);
                Console.ReadLine();
                return false;
            }

            if (times.Length != 3)
            {
                Console.WriteLine("Time Format Exception: " + in_time);
                Console.ReadLine();
                return false;
            }

            if (!int.TryParse(dates[0], out year))
            {
                Console.WriteLine("Date Format Exception: " + in_date);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(dates[1], out month))
            {
                Console.WriteLine("Date Format Exception: " + in_date);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(dates[2], out day))
            {
                Console.WriteLine("Date Format Exception: " + in_date);
                Console.ReadLine();
                return false;
            }

            if (!int.TryParse(times[0], out hour))
            {
                Console.WriteLine("Date Format Exception: " + in_time);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(times[1], out minute))
            {
                Console.WriteLine("Date Format Exception: " + in_time);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(times[2], out second))
            {
                Console.WriteLine("Date Format Exception: " + in_time);
                Console.ReadLine();
                return false;
            }

            date = new DateTime(year, month, day, hour, minute, second);
            return true;
        }

        private static void computeDistance(SortedDictionary<int, SortedDictionary<DateTime, Position>> userPositions)
        {
            Console.WriteLine("Start!");

            for (int k = 1; k < 33; k++)
            {
                SortedDictionary<DateTime, Position> u1 = userPositions[k];
                string u1string;
                if (k < 10)
                    u1string = "0" + k;
                else
                    u1string = k.ToString();

                StreamWriter writer = new StreamWriter(string.Format("D:\\Work\\Pervasive\\Transportation\\output\\{0}.txt", u1string));
                StreamWriter wX = new StreamWriter(string.Format("D:\\Work\\Pervasive\\Transportation\\output\\{0}x.txt", u1string));
                StreamWriter wY = new StreamWriter(string.Format("D:\\Work\\Pervasive\\Transportation\\output\\{0}y.txt", u1string));
                for (int i = 1; i < 33; i++)
                {
                    if (i == k){
                        writer.WriteLine("----");
                        continue;
                    }
                   
                    SortedDictionary<DateTime, Position> u2 = userPositions[i];

                    SortedDictionary<DateTime, Position>.Enumerator u1et = u1.GetEnumerator();
                    SortedDictionary<DateTime, Position>.Enumerator u2et = u2.GetEnumerator();

                    doCommon(writer, wX, wY, u1et, u2et, i, u1.Count, u2.Count);
                }
                wX.Close();
                wY.Close();
                writer.Close();
            }
        }

        private static void parseLabelDate(string date, out int year, out int month, out int day)
        {
            string[] splits = date.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
            year = int.Parse(splits[0]);
            month = int.Parse(splits[1]);
            day = int.Parse(splits[2]);
        }

        private static void parseLabelTime(string time, out int hour, out int minute, out int second)
        {
            string[] splits = time.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);

            hour = int.Parse(splits[0]);
            minute = int.Parse(splits[1]);
            second = int.Parse(splits[2].Substring(0, 2));
        }
        
        private static double cosineSimilarity(List<double> veca, List<double> vecb)
        {
            double scalea = 0, scaleb = 0;
            double crossproduct = 0;
            for (int i = 0; i < veca.Count; i++)
            {
                crossproduct += veca[i] * vecb[i];

                scalea += veca[i] * veca[i];
                scaleb += vecb[i] * vecb[i];
            }
            double vv = crossproduct / (Math.Sqrt(scalea) * Math.Sqrt(scaleb));            
            return vv;
        }

        #region Label Correlation

        public static void OutputLabelCor()
        {
            calcCosDis();
            calcEuc();
            calcLabelWSCS();
            General.OutputAvgEuc(label_wscsmatrix, null);
            General.OutputAverage(WSCSMatrix, label_wscsmatrix);
            General.OutputFivePercent(label_wscsmatrix);
            General.OutputLabelCor(WSCSMatrix, label_wscsmatrix);
        }

        private static Dictionary<int, Dictionary<int, double>> label_wscsmatrix;
        private static Dictionary<int, Dictionary<int, double>> cosDis;
        private static Dictionary<int, Dictionary<int, double>> eucLid;

        private static void calcLabelWSCS()
        {
            #region idf
            Dictionary<int, double> label_idf;
        
            label_idf = new Dictionary<int, double>();

            foreach (string l in labels)
            {
                double div = 0;
                foreach (int user in label_tf.Keys)
                {
                    if (label_tf[user][label2VectorIndex[l]] != 0)
                        div++;
                }
                label_idf.Add(label2VectorIndex[l], Math.Log((double)label_tf.Keys.Count / div));
            }
            #endregion

            #region label wscs

            label_wscsmatrix = new Dictionary<int, Dictionary<int, double>>();
            foreach (int usera in label_tf.Keys)
            {
                List<double> vectora = new List<double>();
                foreach (int i in label2VectorIndex.Values)                    
                    vectora.Add(label_tf[usera][i] * label_idf[i]);

                label_wscsmatrix.Add(usera, new Dictionary<int, double>());
                foreach (int userb in label_tf.Keys)
                {
                    List<double> vectorb = new List<double>();
                    foreach (int i in label2VectorIndex.Values)
                        vectorb.Add(label_tf[userb][i] * label_idf[i]);

                    label_wscsmatrix[usera].Add(userb, cosineSimilarity(vectora, vectorb));
                }
                label_wscsmatrix[usera][usera] = 0;
            }
            #endregion
        }

        private static void calcCosDis()
        {
            cosDis = new Dictionary<int, Dictionary<int, double>>();

            foreach (int userA in label_tf.Keys)
            {
                cosDis.Add(userA, new Dictionary<int, double>());

                foreach (int userB in label_tf.Keys)
                {
                    double dotProduct = 0;
                    double magA = 0, magB = 0;
                    foreach (var pair in label_tf[userA])
                    {
                        dotProduct += pair.Value * label_tf[userB][pair.Key];
                        magA += pair.Value * pair.Value;
                        magB += label_tf[userB][pair.Key] * label_tf[userB][pair.Key];
                    }
                    cosDis[userA].Add(userB, dotProduct / Math.Sqrt(magA * magB));
                }
            }
        }

        private static void calcEuc()
        {
            eucLid = new Dictionary<int, Dictionary<int, double>>();

            foreach (int userA in label_tf.Keys)
            {
                eucLid.Add(userA, new Dictionary<int, double>());

                foreach (int userB in label_tf.Keys)
                {
                    double sum = 0;
                    foreach (var pair in label_tf[userA])
                    {
                        sum += (pair.Value - label_tf[userB][pair.Key]) * (pair.Value - label_tf[userB][pair.Key]);
                    }
                    eucLid[userA].Add(userB, Math.Sqrt(sum));
                }
            }
        }
        
        #endregion

        #endregion

        #region Trannsportation Output

        private static Dictionary<int, List<string>> allNodesName = new Dictionary<int, List<string>>();

        public static void Output()
        {
            outputSchemaFile();
            outputTruthKnownDataFile();
            outputEdgeFile();
        }

        private static void outputSchemaFile()
        {
            string schemafilePath = root + "\\trans-schema.arff";
            StreamWriter writer = new StreamWriter(schemafilePath);

            writer.WriteLine("@nodetype Position");
            writer.WriteLine("@attribute Name KEY");
            for (int i = 0; i < featVecLen; i++)
                writer.WriteLine("@attribute Speed" + i + " CONTINUOUS");
            writer.WriteLine("@attribute TransType CATEGORICAL");
            writer.WriteLine("@nodedata trans-truth.csv");

            writer.WriteLine();
            writer.WriteLine("@edgetype Linked Position Position");
            writer.WriteLine("@Reversible");
            writer.WriteLine("@edgedata trans-link.rn");
            writer.Close();
        }

        private static void outputTruthKnownDataFile()
        {
            string nodeTruthFilePath = root + "\\trans-truth.csv";
            string nodeKnownFilePath = root + "\\trans-known.csv";
            StreamWriter truthWriter = new StreamWriter(nodeTruthFilePath);
            StreamWriter knownWriter = new StreamWriter(nodeKnownFilePath);

            int maxOutputNumber = 40;
            Random rand = new Random(DateTime.Now.Second);
            //Console.WriteLine(rand.NextDouble().ToString());
            foreach (int user in allUserSpeed.Keys)
            {
                allNodesName.Add(user, new List<string>());

                SortedDictionary<DateTime, SortedList<DateTime, double>> singleUserTrace = allUserSpeed[user];
                
                int instanceID = 0;
                string nodeName = "";

                foreach (DateTime time in singleUserTrace.Keys)
                {
                    SortedList<DateTime, double> singleTrace = singleUserTrace[time];                    
                    if (singleTrace.Count < 1) continue;

                    string label = "";
                    int year = singleTrace.First().Key.Year, month = singleTrace.First().Key.Month;
                    SortedDictionary<int, SortedDictionary<int, SortedList<DateTime, LabelInfo>>> userLabel = allUserLabel[user];
                    if (!userLabel.Keys.Contains(year))
                        continue;
                    if (!userLabel[year].Keys.Contains(month))
                        continue;

                    IEnumerator<KeyValuePair<DateTime,double>> traceEnum = singleTrace.GetEnumerator();
                    IEnumerator<KeyValuePair<DateTime, LabelInfo>> labelEnum = userLabel[year][month].GetEnumerator();
                    bool labelMove = labelEnum.MoveNext(), traceMove = traceEnum.MoveNext();
                    int featureLength = 0;
                    string line;
                    while (traceMove)
                    {
                        while (labelMove &&
                            (labelEnum.Current.Value.startTime > traceEnum.Current.Key || labelEnum.Current.Value.endTime < traceEnum.Current.Key))
                            labelMove = labelEnum.MoveNext();
                        if (!labelMove)
                            break;
                        
                        label = labelEnum.Current.Value.label;
                        featureLength = 0;
                        line = "";

                        do
                        {
                            line = line + traceEnum.Current.Value.ToString("0.0000") + ",";

                            featureLength++;
                            traceMove = traceEnum.MoveNext();
                        } while (featureLength < featVecLen && traceMove && traceEnum.Current.Key >= labelEnum.Current.Value.startTime
                            && traceEnum.Current.Key <= labelEnum.Current.Value.endTime);

                        if (featureLength >= featVecLen)
                        {
                            if (label == "walk" && rand.NextDouble() > 0.3)
                                continue;
                            if (label == "bike" && rand.NextDouble() > 0.42)
                                continue;
                            if (label != "walk" && label != "bike" && rand.NextDouble() > 0.99)
                                continue;
                            //if (rand.NextDouble() < 0.5) continue;

                            nodeName = "U" + user + "F" + instanceID;
                            allNodesName[user].Add(nodeName);

                            line = nodeName + "," + line + label;
                            truthWriter.WriteLine(line);

                            if (rand.NextDouble() <= knownRatio) knownWriter.WriteLine(nodeName + "," + label);

                            instanceID++;

                            if (instanceID >= maxOutputNumber)
                                break;
                        }
                    }

                    if (instanceID >= maxOutputNumber)
                        break;
                    //if (!findLabel(user, singleTrace, out label))
                    //    continue;

                    //nodeName = "U" + user + "S" + instanceID;
                    //allNodesName[user].Add(nodeName);                    
                    //writer.Write(nodeName);
                    //foreach (double speed in singleTrace.Values)
                    //    writer.Write("," + speed.ToString("0.0000"));
                    //writer.WriteLine("," + label);
                    
                    //instanceID++;
                    //if (instanceID >= maxOutputNumber)
                    //    break;
                }               
            }           
            truthWriter.Close();
            knownWriter.Close();
        }

        private static void outputEdgeFile()
        {
            string edgeFilePath = root + "\\trans-link.rn";

            StreamWriter writer = new StreamWriter(edgeFilePath);            
            foreach (int userA in allNodesName.Keys)
            {
                List<string> nodesA = allNodesName[userA];
                foreach (int userB in allNodesName.Keys)
                {
                    if (userB <= userA)
                        continue;                    

                    List<string> nodesB = allNodesName[userB];

                    double linkStrength = WSCSMatrix[userA][userB];
                    if (linkStrength < 0.00005)
                        continue;

                    foreach (string nodeA in nodesA)
                        foreach (string nodeB in nodesB)
                            writer.WriteLine(nodeA + "," + nodeB + "," + linkStrength.ToString("0.0000"));
                }
            }
            writer.Close();
        }

        private static bool findLabel(int user,SortedList<DateTime, double> singleTrace, out string label)
        {
            label = "";
            if (singleTrace.Count < 1) return false;
            int year = singleTrace.First().Key.Year, month = singleTrace.First().Key.Month;
            SortedDictionary<int, SortedDictionary<int, SortedList<DateTime, LabelInfo>>> userLabel = allUserLabel[user];
            if (!userLabel.Keys.Contains(year))
                return false;
            if (!userLabel[year].Keys.Contains(month))
                return false;

            foreach (LabelInfo l in userLabel[year][month].Values)
                if (l.startTime <= singleTrace.First().Key && l.endTime >= singleTrace.Last().Key)
                {
                    label = l.label;
                    return true;
                }

            return false;
        }

        #endregion

        #region Transportation Speed

        static DateTime standardTime = new DateTime(1900, 01, 01, 0, 0, 0);
        static string sFolderPath = root + @"\Speed\";

        public static void ReadInSpeed()
        {
            allUserSpeed.Clear();
            List<string> subFolders = getTransSubFoleders();
            DateTime st1 = DateTime.Now;
            foreach(string subFolder in subFolders)
            {
                int user = int.Parse(subFolder);
                allUserSpeed.Add(user, new SortedDictionary<DateTime, SortedList<DateTime, double>>());
                StreamReader reader = new StreamReader(sFolderPath + user + "_Speed.txt");
                string line = "";
                while ((line = reader.ReadLine()) != null)
                {
                    string[] splits = line.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
                    DateTime startTime = standardTime.AddSeconds(double.Parse(splits[0]));

                    allUserSpeed[user].Add(startTime, new SortedList<DateTime, double>());
                    for (int i = 1; i < splits.Length; i++)
                    {
                        string[] pair = splits[i].Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
                        DateTime time = startTime.AddSeconds(double.Parse(pair[0]));
                        double speed = double.Parse(pair[1]);
                        allUserSpeed[user][startTime].Add(time, speed);
                    }
                }
                reader.Close();
            }
            DateTime now = DateTime.Now;
            MCBConsole.Trace("Read In Speed Cost : " + (now - st1).TotalSeconds);
        }

        private static void OutputSpeed()
        {   
            foreach (int user in allUserSpeed.Keys)
            {
                StreamWriter writer = new StreamWriter(sFolderPath + user + "_Speed.txt");
                SortedDictionary<DateTime, SortedList<DateTime, double>> singleUser = allUserSpeed[user];

                foreach (DateTime startTime in singleUser.Keys)
                {
                    SortedList<DateTime, double> singleTrace = singleUser[startTime];
                    if (singleTrace.Count < featVecLen)
                        continue;
                    writer.Write((startTime - standardTime).TotalSeconds);
                    foreach (KeyValuePair<DateTime, double> pair in singleTrace)                    
                        writer.Write(";" + (pair.Key - startTime).TotalSeconds + "," + pair.Value.ToString("0.00000"));
                    writer.WriteLine();
                }
                writer.Close();
            }            
        }

        #endregion

        #region T-Drive

        public static void ReadTDrive()
        {
            DateTime ST = DateTime.Now;
            MCBConsole.Trace("Start : " + ST);
            DateTime NOW;

            string root = "D:\\Work\\Pervasive\\T-Drive";

            List<string> subFolders = getTDriveSubFolders();

            SortedDictionary<int, SortedDictionary<DateTime, Position>> allUserPositions = new SortedDictionary<int, SortedDictionary<DateTime, Position>>();
            int count = 0, total = 0;
            foreach (string subfolder in subFolders)
            {
                string path = root + "\\" + subfolder;
                IEnumerable<string> files = System.IO.Directory.EnumerateFiles(path);
                IEnumerator<string> fit = files.GetEnumerator();

                int userID;
                DateTime datetime;
                double latitude;
                double longitude;

                while (fit.MoveNext())
                {
                    string filename = fit.Current;

                    StreamReader reader = new StreamReader(filename);

                    string line = "";
                    SortedDictionary<DateTime, Position> singleUser;

                    while ((line = reader.ReadLine()) != null)
                    {
                        string[] splits = line.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);

                        if (splits.Length != 4)
                            continue;

                        if (!int.TryParse(splits[0], out userID))
                            continue;

                        if (!tryParseDate(splits[1], out datetime))
                            continue;

                        if (!double.TryParse(splits[2], out latitude))
                            continue;

                        if (!double.TryParse(splits[3], out longitude))
                            continue;
                        
                        if (!allUserPositions.TryGetValue(userID,out singleUser)){
                            singleUser = new SortedDictionary<DateTime, Position>();
                            allUserPositions.Add(userID, singleUser);
                        }
                        Position p;
                        if (!singleUser.TryGetValue(datetime, out p))
                        {
                            singleUser.Add(datetime, new Position(latitude, longitude));
                            count++;
                            allUserPositions[userID] = singleUser;
                        }
                    }
                }

                NOW = DateTime.Now;
                TimeSpan dur = NOW - ST;
                Console.WriteLine(subfolder + " count: " + count + "  " + dur.ToString());
                total += count;
                count = 0;                
            }
            Console.WriteLine("Total : " + total);
            getColocate(allUserPositions);
        }

        private static void getColocate(SortedDictionary<int, SortedDictionary<DateTime, Position>> userPositions)
        {
            Console.WriteLine("Start!");
            SortedDictionary<int, SortedDictionary<DateTime, Position>>.Enumerator enum1 = userPositions.GetEnumerator();            

            while (enum1.MoveNext())
            {
                SortedDictionary<DateTime, Position> u1 = enum1.Current.Value;

                string u1string;
                if (enum1.Current.Key < 10)
                    u1string = "0" + enum1.Current.Key;
                else
                    u1string = enum1.Current.Key.ToString();

                StreamWriter writer = new StreamWriter(string.Format("D:\\Work\\Pervasive\\T-Drive\\output\\{0}.txt", u1string));
                StreamWriter wX = new StreamWriter(string.Format("D:\\Work\\Pervasive\\T-Drive\\output\\{0}x.txt", u1string));
                StreamWriter wY = new StreamWriter(string.Format("D:\\Work\\Pervasive\\T-Drive\\output\\{0}y.txt", u1string));
                SortedDictionary<int, SortedDictionary<DateTime, Position>>.Enumerator enum2 = userPositions.GetEnumerator();
                while (enum2.MoveNext())
                {
                    if (enum2.Current.Key == enum1.Current.Key)
                    {
                        writer.WriteLine("----");
                        continue;
                    }

                    SortedDictionary<DateTime, Position> u2 = enum2.Current.Value;

                    SortedDictionary<DateTime, Position>.Enumerator u1et = u1.GetEnumerator();
                    SortedDictionary<DateTime, Position>.Enumerator u2et = u2.GetEnumerator();

                    doCommon(writer, wX, wY, u1et, u2et, enum2.Current.Key, u1.Count, u2.Count);
                }
                writer.Close();
                wX.Close();
                wY.Close();
            }
        }

        private static List<string> getTDriveSubFolders()
        {
            List<string> subs = new List<string>();

            for (int i = 1; i < 15; i++)
                if (i < 10)
                    subs.Add("0" + i);
                else
                    subs.Add(i.ToString());

            return subs;
        }

        private static bool tryParseDate(string in_datetime, out DateTime date)
        {
            string[] splits = in_datetime.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);

            if (splits.Length != 2)
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                date = DateTime.Now;
                return false;
            }

            string[] dates = splits[0].Split(new char[] { '-' }, StringSplitOptions.RemoveEmptyEntries);
            string[] times = splits[1].Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries);

            int year, month, day;
            int hour, minute, second;
            date = DateTime.Now;

            if (dates.Length != 3)
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }

            if (times.Length != 3)
            {
                Console.WriteLine("Time Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }

            if (!int.TryParse(dates[0], out year))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(dates[1], out month))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(dates[2], out day))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }

            if (!int.TryParse(times[0], out hour))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(times[1], out minute))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }
            if (!int.TryParse(times[2], out second))
            {
                Console.WriteLine("Date Format Exception: " + in_datetime);
                Console.ReadLine();
                return false;
            }

            date = new DateTime(year, month, day, hour, minute, second);
            return true;
        }
        
        #endregion

        #region All

        private static double CalcDistance(Position p1, Position p2)
        {
            double R = 6367000, r = 4877405;
            double l1 = Math.PI * R * Math.Abs(p1.latitude - p2.latitude) / 180;
            double l2 = Math.PI * r * Math.Abs(p1.longitude - p2.longitude) / 180;
            return Math.Sqrt(l1 * l1 + l2 * l2);
        }

        private static void doCommon(StreamWriter writer, StreamWriter wX, StreamWriter wY, SortedDictionary<DateTime, Position>.Enumerator u1et, 
            SortedDictionary<DateTime, Position>.Enumerator u2et,int u2ID,int u1num,int u2num )
        {
            SortedList<DateTime, double> distances = new SortedList<DateTime, double>();

            bool u1move = u1et.MoveNext(), u2move = u2et.MoveNext();
            TimeSpan dur;

            while (u1move && u2move)
            {
                dur = u1et.Current.Key - u2et.Current.Key;
                double durseconds = dur.TotalSeconds;

                if (Math.Abs(durseconds) < timethred)
                {
                    double dis = CalcDistance(u1et.Current.Value, u2et.Current.Value);
                    distances.Add(u1et.Current.Key - new TimeSpan(dur.Ticks / 2), dis);

                    u1move = u1et.MoveNext();
                    u2move = u2et.MoveNext();
                }
                else if (durseconds > 0)
                    u2move = u2et.MoveNext();
                else if (durseconds < 0)
                    u1move = u1et.MoveNext();
            }

            int durnum = 0, disnear = 0, timenear = 0;

            DateTime start = DateTime.Now, last = DateTime.Now;
            bool colocate = false;

            List<double> durations = new List<double>();
            foreach (KeyValuePair<DateTime, double> pair in distances)
            {
                timenear++;
                if (pair.Value < disthred)
                {
                    disnear++;
                    if (colocate)
                    {
                        if ((pair.Key - last).TotalSeconds <= gracePeriod)
                            last = pair.Key;
                        else
                        {
                            double t = (last - start).TotalSeconds;
                            if (t > ignorePeriod)
                                durations.Add(t);

                            start = last = pair.Key;
                        }
                    }
                    else
                    {
                        colocate = true;
                        start = last = pair.Key;
                    }
                }
                else
                {
                    if (colocate)
                    {
                        if (!((pair.Key - last).TotalSeconds <= gracePeriod))
                        {
                            double t = (last - start).TotalSeconds;

                            if (t > ignorePeriod)
                                durations.Add(t);

                            colocate = false;
                        }
                    }
                    else
                    {
                        // do nothing
                    }
                }
            }
            if (colocate)
            {
                double t = (last - start).TotalSeconds;
                if (t > ignorePeriod)
                    durations.Add(t);
            }

            foreach (double duration in durations)
            {
                wX.Write(duration + " ");
                wY.Write(u2ID + " ");
            }
            if (durations.Count == 0)
            {
                wX.Write(0 + " ");
                wY.Write(u2ID + " ");
            }

            durnum = durations.Count;
            writer.WriteLine(durnum + " / " + disnear + " / " + timenear + " / " + u1num + " " + u2num);
        }

        private static List<string> getTransSubFoleders()
        {
            List<string> subs = new List<string>();
            for (int i = 1; i < 33; i++)
                if (i < 10)
                    subs.Add("00" + i);
                else
                    subs.Add("0" + i);

            return subs;
        }

        struct Position
        {
            public Position(double lat, double longi,double alti,bool n)
            {
                latitude = lat;
                longitude = longi;
                altitude = alti;
                normal = n;
            }

            public Position(double lat, double longi)
            {
                latitude = lat;
                longitude = longi;
                altitude = -1;
                normal = false;
            }

            public double latitude;
            public double longitude;
            public double altitude;
            public bool normal;
        }

        struct LabelInfo
        {
            public DateTime startTime;
            public DateTime endTime;
            public string label;

            public LabelInfo(DateTime s, DateTime e,string l)
            {
                startTime = s;
                endTime = e;
                label = l;
            }
        }

        #endregion
    }
};